/*
 * tf/idf implementation 
 * Author: Thanh Dao, thanh.dao@gmx.net
 */
using System;
using System.Collections;
using System.Collections.Generic;


namespace ServiceRanking
{
    /// <summary>
    /// Summary description for TF_IDFLib.
    /// </summary>
    public class TFIDFMeasure
    {
        private string[] _docs;
        private string[][] _ngramDoc;
        private int _numDocs = 0;
        public int _numTerms = 0;
        private ArrayList _terms;

        private List<LargeIntList> _termFreq;//
        private List<LargeFloatList> _termWeight;//

        private int[] _maxTermFreq;
        private int[] _docFreq;

        public delegate void OnSEProgress(string msg);
        public OnSEProgress OnProgressStart, OnProgressEnd;

        public delegate void OnProgress(float total, float count);
        public OnProgress OnProgressEvent;

        public ArrayList GetTerms()
        {
            return _terms;
        }

        public class TermVector
        {
            public static float ComputeCosineSimilarity(float[] vector1, float[] vector2)
            {
                if (vector1.Length != vector2.Length)
                    throw new Exception("DIFER LENGTH");


                float denom = (VectorLength(vector1) * VectorLength(vector2));
                if (denom == 0F)
                    return 0F;
                else
                    return (InnerProduct(vector1, vector2) / denom);

            }

            public static float InnerProduct(float[] vector1, float[] vector2)
            {

                if (vector1.Length != vector2.Length)
                    throw new Exception("DIFFER LENGTH ARE NOT ALLOWED");


                float result = 0F;
                for (int i = 0; i < vector1.Length; i++)
                    result += vector1[i] * vector2[i];

                return result;
            }

            public static float VectorLength(float[] vector)
            {
                float sum = 0.0F;
                for (int i = 0; i < vector.Length; i++)
                    sum = sum + (vector[i] * vector[i]);

                return (float)Math.Sqrt(sum);
            }

        }

        private IDictionary _wordsIndex = new Hashtable();

        public TFIDFMeasure()
        { }

        public void Init(string[] documents)
        {
            _docs = documents;
            _numDocs = documents.Length;
            MyInit();
        }

        //private void GeneratNgramText()
        //{	
        //}

        private ArrayList GenerateTerms(string[] docs)
        {
            ArrayList uniques = new ArrayList();
            _ngramDoc = new string[_numDocs][];
            if (OnProgressStart != null)
                OnProgressStart("GenerateTerms");
            for (int i = 0; i < docs.Length; i++)
            {
                Tokeniser tokenizer = new Tokeniser();
                string[] words = tokenizer.Partition(docs[i]);
                if (OnProgressEvent != null)
                    OnProgressEvent(docs.Length, i);
                for (int j = 0; j < words.Length; j++)
                    if (!uniques.Contains(words[j]))
                        uniques.Add(words[j]);

            }
            if (OnProgressEnd != null)
                OnProgressEnd("GenerateTerms");
            return uniques;
        }

        private static object AddElement(IDictionary collection, object key, object newValue)
        {
            object element = collection[key];
            collection[key] = newValue;
            return element;
        }

        public int GetTermIndex(string term)
        {
            object index = _wordsIndex[term];
            if (index == null) return -1;
            return (int)index;
        }

        private void MyInit()
        {
            _terms = GenerateTerms(_docs);
            _numTerms = _terms.Count;

            _maxTermFreq = new int[_numDocs];
            _docFreq = new int[_numTerms];
            _termFreq = new List<LargeIntList>();//_numTerms
            _termWeight = new List<LargeFloatList>();//_numTerms
            if (OnProgressStart != null)
                OnProgressStart("Loading terms");
            for (int i = 0; i < _terms.Count; i++)
            {
                if (OnProgressEvent != null)
                    OnProgressEvent(_terms.Count, i);
                LargeFloatList tw = new LargeFloatList();//_numDocs
                LargeIntList tf = new LargeIntList();//_numDocs
                for (int j = 0; j < _numDocs; j++)
                {
                    tw.Add(0);
                    tf.Add(0);
                }
                _termWeight.Add(tw);
                _termFreq.Add(tf);

                AddElement(_wordsIndex, _terms[i], i);
            }

            if (OnProgressEnd != null)
                OnProgressEnd("Loading terms");

            GenerateTermFrequency();
            GenerateTermWeight();

        }

        private float Log(float num)
        {
            return (float)Math.Log(num);//log2
        }

        private void GenerateTermFrequency()
        {
            if (OnProgressStart != null)
                OnProgressStart("GenerateTermFrequency");
            for (int i = 0; i < _numDocs; i++)
            {
                if (OnProgressEvent != null)
                    OnProgressEvent(_numDocs, i);

                string curDoc = _docs[i];
                IDictionary freq = GetWordFrequency(curDoc);
                IDictionaryEnumerator enums = freq.GetEnumerator();
                _maxTermFreq[i] = int.MinValue;
                while (enums.MoveNext())
                {
                    string word = (string)enums.Key;
                    int wordFreq = (int)enums.Value;
                    int termIndex = GetTermIndex(word);

                    _termFreq[termIndex][i] = wordFreq;
                    _docFreq[termIndex]++;

                    if (wordFreq > _maxTermFreq[i]) _maxTermFreq[i] = wordFreq;
                }
            }
            if (OnProgressEnd != null)
                OnProgressEnd("GenerateTermFrequency");
        }


        private void GenerateTermWeight()
        {
            if (OnProgressStart != null)
                OnProgressStart("GenerateTermWeight");
            for (int i = 0; i < _numTerms; i++)
            {
                if (OnProgressEvent != null)
                    OnProgressEvent(_numTerms, i);
                for (int j = 0; j < _numDocs; j++)
                    _termWeight[i][j] = ComputeTermWeight(i, j);
            }
            if (OnProgressEnd != null)
                OnProgressEnd("GenerateTermWeight");
        }

        public float GetTermFrequency(int term, int doc)
        {
            int freq = _termFreq[term][doc];
            int maxfreq = _maxTermFreq[doc];

            return ((float)freq / (float)maxfreq);
        }

        public float GetInverseDocumentFrequency(int term)
        {
            int df = _docFreq[term];
            return Log((float)(_numDocs) / (float)df);
        }

        private float ComputeTermWeight(int term, int doc)
        {
            float tf = GetTermFrequency(term, doc);
            float idf = GetInverseDocumentFrequency(term);
            return tf * idf;
        }

        public float[] GetTermVector(int doc)
        {
            float[] w = new float[_numTerms];
            for (int i = 0; i < _numTerms; i++)
                w[i] = _termWeight[i][doc];
            return w;
        }

        public float GetSimilarity(int doc_i, int doc_j)
        {
            float[] vector1 = GetTermVector(doc_i);
            float[] vector2 = GetTermVector(doc_j);

            return TermVector.ComputeCosineSimilarity(vector1, vector2);

        }

        private IDictionary GetWordFrequency(string input)
        {
            string convertedInput = input.ToLower();

            Tokeniser tokenizer = new Tokeniser();
            String[] words = tokenizer.Partition(convertedInput);
            Array.Sort(words);

            String[] distinctWords = GetDistinctWords(words);

            IDictionary result = new Hashtable();
            for (int i = 0; i < distinctWords.Length; i++)
            {
                object tmp;
                tmp = CountWords(distinctWords[i], words);
                result[distinctWords[i]] = tmp;

            }

            return result;
        }

        private string[] GetDistinctWords(String[] input)
        {
            if (input == null)
                return new string[0];
            else
            {
                ArrayList list = new ArrayList();

                for (int i = 0; i < input.Length; i++)
                    if (!list.Contains(input[i])) // N-GRAM SIMILARITY?				
                        list.Add(input[i]);

                return Tokeniser.ArrayListToArray(list);
            }
        }

        private int CountWords(string word, string[] words)
        {
            int itemIdx = Array.BinarySearch(words, word);

            if (itemIdx > 0)
                while (itemIdx > 0 && words[itemIdx].Equals(word))
                    itemIdx--;

            int count = 0;
            while (itemIdx < words.Length && itemIdx >= 0)
            {
                if (words[itemIdx].Equals(word)) count++;

                itemIdx++;
                if (itemIdx < words.Length)
                    if (!words[itemIdx].Equals(word)) break;

            }

            return count;
        }
    }
}
